#!/usr/bin/python
#coding=utf-8
import logging
import urlparse
import re
import sys
import datetime
from lxml import etree
import os
from math import ceil
from PostGetter import PostGetter,htmlentitydecode,chkLogin
import _mysql_exceptions

class CdPostGetter(PostGetter):
	def __init__(self,cookie_file):
		PostGetter.__init__(self,cookie_file)
		self.NR_REPLY_PER_PAGE=40

	def login(self):
		self.logger.debug('to login ...')
		data={}
		forum=Forum.objects.filter(name=self.forum_name).get()
		if forum:
			data['loginfield']='username'
			data['username']=forum.usr.encode(self.dft_html_encoding)
			data['password']=forum.pwd.encode(self.dft_html_encoding)
			data['cookietime']='checked'
			data['questionid']='0'
			r,_,_=self._getResponse(forum.login_url.encode(self.dft_html_encoding),data)
			if not r:
				self._signed_in=False
			else:
				r=self._tryDecode(r)
##				open('/home/kevin/tmp.html','w').write(r)

				# check html or url
				if r.find(u'欢迎您回来')!=-1:
					self.cj.save(self.cookie_file)
					self._signed_in=True
				else:
					self.logger.info('login failed!')
		else:
			self.logger.debug('can\'t find forum %s, can\'t do login !',self.name)

		return self._signed_in

	@chkLogin
	def getPostList(self,forum_name,sector_name,time_since,page_start,page_stop):
		self.forum_name=forum_name

		upd=None
		cnt=0
		try:
			forum=Forum.objects.filter(name=forum_name)[0:1].get()
		except Forum.DoesNotExist:
			self.logger.debug('can\'t find forum info of %s!',forum_name)
			raise

		try:
			sector=Sector.objects.filter(forum=forum,name=sector_name)[0:1].get()
		except Sector.DoesNotExist:
			self.logger.debug('can\'t find sector info of %s!',sector_name)
			raise

		for i in xrange(page_start,page_stop): # max page number
			pageurl=sector.url%(i,)
##			self.logger.info('get %s ...',pageurl)
			data=self._getData(pageurl,None,'%s-page %02d'%(sector.name,i))
			if not data:
				self.logger.debug('got nothing, skip!')
				break


			# 用 lxml xpath 选择包含正文的 div
			parser=etree.HTMLParser()
			tree=etree.fromstring(data,parser)
##			el=tree.xpath(sector.url_pattern)
##			el=tree.xpath('//div[@id="threadlist"]//form[@id="moderate"]/table/*[not(@id="separatorline")]')
			el=tree.xpath('//div[@id="threadlist"]//form[@id="moderate"]/table/*[@id]')

			sticktopic=None
			for item in el: # 列表中的每个主题
				if item.attrib['id']=='separatorline':
					continue

##				self.logger.debug('-='*10)
				if 'stickthread' in item.xpath('@id')[0]:
					sticktopic=True
				else:
					sticktopic=False

				title=item.xpath('./tr/th/a/text()')[0]

				if item.xpath('./tr/th/span[@class="tps"]'):
					pages=int(item.xpath('./tr/th/span[@class="tps"]/a[position()=last()]/text()')[0])
				else:
					pages=1

				# 取主题信息
##				title_url=item.xpath('./tr/th/span[1]/a[1]/@href')[0]
				title_url=item.xpath('./tr/th/a/@href')[0]
				m=re.search('thread-(\d+)-.*?.html',title_url)
				if m:
					lid=m.group(1)
				else:
					self.logger.debug('can\'t find locate_id for post %s',title)
					lid=''

				author=item.xpath('./tr/td[@class="by"][1]/cite/a/text()')[0]
				assert author


				crt_date=item.xpath('./tr/td[@class="by"][1]/em/span/text()')[0]
				unknownreply=True
				try:
					nr_reply=int(item.xpath('./tr/td[@class="num"]/a/text()')[0])
					unknownreply=False
				except ValueError:
##					self.logger.debug('post %s has no reply number!',lid) # 移动/锁贴
					nr_reply=0

				upd_date=item.xpath('./tr/td[@class="by"][2]/em/a/text()')[0]
##				self.logger.info('(%02d) (%02d) %s |%s|%s|%s|%s|%s',cnt,pages,title,author,crt_date,upd_date,nr_reply,title_url)
				cnt+=1
##				self.logger.info('%d---%02d)%s|%s|%d-%d|%s|%s',item.sourceline,cnt,lid,upd_date,nr_reply,pages,author,title)
				self.logger.info('%s\n%02d)\t%s|%s|%s-%d|%s|%s', '-='*10, cnt, lid, upd_date, str(nr_reply) if not unknownreply else '?', pages, author, title.strip())
				upd=datetime.datetime.strptime(upd_date,'%Y-%m-%d %H:%M')
				if time_since and (not sticktopic) and upd < time_since:
					self.logger.info('%s\ngot time stop in page list.','-~'*20)
					return

				# check DB, insert if not exist, else check update
				try:
					p=Post.objects.filter(sector=sector,locate_id=lid)[0:1].get()
				except Post.DoesNotExist:
					p=Post(sector=sector,locate_id=lid,url=title_url,title=title,author=author,
						crt_date=crt_date,upd_date=upd_date,nr_reply=0)
					p.save()
					p.nr_reply=nr_reply
					self.logger.debug('post %s created.',lid)
					self.getPost(p)
				else:
					if p.upd_date!=upd:
						p.upd_date=upd_date
						p.save()
					if unknownreply: # check every time 不过每次最多只能获取一页新帖子
						self.logger.info('post %s nr_reply may changed(unknown reply cnt). now %d in DB',lid,p.nr_reply+1)
						nr_reply=p.nr_reply+1 # 设为比原来的大以触发查找

					if p.nr_reply!=nr_reply:
						self.logger.info('post %s nr_reply changed. %+d',lid,nr_reply-p.nr_reply)
						self.getPost(p,nr_reply)

	def getPost(self,post,new_nr=0):
		cnt=0
		sr=urlparse.urlsplit(post.url)
		if not sr.scheme:
			posturl=post.sector.base_url+post.url
		else:
			posturl=post.url

		posturl='%d'.join(re.findall('(.*?thread-\d+-)\d+(-.*?.html)',posturl)[0])

		startpage,stoppage=self.getPageRange(self.NR_REPLY_PER_PAGE,post.nr_reply,new_nr)

		self.logger.debug('post %s page range [%d,%d)',post.locate_id,startpage,stoppage)
		parser=etree.HTMLParser()
		for pg in xrange(startpage,stoppage):
			self.logger.debug('post %s %s ...',post.locate_id,posturl%pg)
			data=self._getData(posturl%pg,None,post.title)
			if not data:
				self.logger.debug('-=-=- !!!!!!!!!!!!!! got None or \'\', skip !!!!!!!!!!!!!!!!!!! -=-=-')
				continue
			# check authentication. some posts need high permission to view.
			if data.find('超级大本营军事论坛 提示信息')!=-1 and data.find('您无权进行当前操作，原因如下：')!=-1:
				self.logger.debug('got err %s',re.search('<div class="alert_error">(.*?)</div>',data,re.M|re.S|re.I).group(1))
				if self.login():
					data=self._getData(posturl%pg,None,post.title)
					if not data:
						self.logger.debug('-=-=- !!!!!!!!!!!!!! got None or \'\', skip !!!!!!!!!!!!!!!!!!! -=-=-')
						continue


			tree=etree.fromstring(data,parser)

			posts=tree.xpath('//div[@id="postlist"]/div[starts-with(@id,"post_")=true()]')
			for item in posts:
				replyid=item.attrib['id'][5:]
				assert replyid
				try:
					author=item.xpath('./table//td[@class="pls"]/div[@class="pi"]/div[@class="authi"]/a/text()')[0] # item.xpath('./table/tr[1]/td[@class="pls"]/div[@class="pi"]/div[@class="authi"]/a/text()')[0]
				except IndexError:
					author=item.xpath('./table//td[@class="pls"]/div[@class="pi"]/text()')[0].strip()
				assert author is not None

##				crt_date=item.xpath('./table/tr[1]/td[@class="postcontent"]/div[@class="postinfo"]/div[@class="posterinfo"]/div[@class="authorinfo"]/em/text()')[0][4:]
				crt_date=item.xpath('./table//div[@class="pti"]/div[@class="authi"]/em/text()')[0][4:] # crt_date=item.xpath('./table/tr[1]/td[@class="plc"]/div[@class="pi"]/div[@class="pti"]/div[@class="authi"]/em/text()')[0][4:]

				# './table/tr[1]/td[@class="plc"]/div[@class="pct"]/div[@class="pcb"]/div[@class="t_fsz"]/table/tr/td[@id="postmessage_%s"]'%replyid
##				replycontent= item.xpath('//td[@id="postmessage_%s"]/*[not(@class="a_pr" or @class="pstatus")]'%replyid)[0]
				try:
					replycontent= item.xpath('//td[@id="postmessage_%s"]'%replyid)[0]
				except IndexError:
					if item.xpath('./table//div[@class="pct"]/div[@class="pcb"]/div[@class="locked"]'):
						replycontent= item.xpath('./table//div[@class="pct"]/div[@class="pcb"]/div[@class="locked"]')[0]
						replycontent= htmlentitydecode(etree.tostring(replycontent)).strip()
					else:
						raise
				else:
					assert replycontent is not None
					# remove element 'ad' and 'poststatus'
					for i in replycontent:
						if 'class' in  i.attrib and i.attrib['class'] in ('a_pr','pstatus'):
							textlist=re.findall(r'\A<%s.*?>.*?</%s>(.*?)\Z'%(i.tag,i.tag), etree.tostring(i), re.M|re.S|re.U)
							textlist=[ x for x in textlist if x.strip()!='' ]
							if len(textlist)>0:
								remaintext='<br />'.join(textlist)
								newelement=item.makeelement('br')
								newelement.text=remaintext
								replycontent.replace(i,newelement)
							else:
								replycontent.remove(i)
					replycontent=self.exclude_first_td_tag.match(htmlentitydecode(etree.tostring(replycontent)).strip()).group(1)

				try:
					r=Reply.objects.filter(post=post,locate_id=replyid)[0:1].get()
				except Reply.DoesNotExist:
					r=Reply(post=post,locate_id=replyid,crt_date=crt_date,author=author,content=replycontent)
					try:
						r.save()
					except _mysql_exceptions.Warning:
						self.logger.debug('got _mysql_exceptions.Warning!')
						r.content=self.exclude_first_td_tag.match(etree.tostring(item[1]).strip()[:-5]).group(1)
						if replycontent.startswith('<br/>'):
							replycontent=replycontent[5:]
						r.save()

					cnt+=1


		if new_nr!=0:
			if post.nr_reply+cnt==new_nr:
				self.logger.info('post %s %+d reply. now %d',post.locate_id,cnt,new_nr+1)
				post.nr_reply+=cnt # 增加实际变化（新增）的数量
			else:
				self.logger.debug('post %s %+d reply, %d != expect %d (no right new_nr info?)',post.locate_id,cnt,post.nr_reply+cnt,new_nr)
				# 检查实际获得数量
				actualcnt=Reply.objects.filter(post=post).count()
				self.logger.info('post %s actual %d reply in DB',post.locate_id,actualcnt)
				post.nr_reply=actualcnt-1 if actualcnt-1>=0 else 0
		else:
			if post.nr_reply+1==cnt:
				self.logger.info('post %s init %+d reply.',post.locate_id,cnt)
			else:
				self.logger.info('post %s init %+d reply, != expect %d',post.locate_id,cnt,post.nr_reply+1)
			post.nr_reply=cnt-1 if cnt-1>=0 else 0 # 设为实际获得值-1，以便下次再次尝试查找新增帖子
		post.save()



os.environ['DJANGO_SETTINGS_MODULE']='postgetter.settings'
sys.path.append(os.path.dirname(os.path.abspath(__file__)))
# 重要 为了能使用django的orm模块，需要创建目录btView，在里面放上空
#  文件__init__.py和文件models.py,这样django就会去数据库中找btView_Video,
#  btView_Screenshot，btView是创建的另一个ajango应用的app_name
# 如何单独使用ajango的orm模块见 http://wiki.woodpecker.org.cn/moin/UsingDjangoAsAnStandaloneORM
from postgetter.getpost.models import Forum,Sector,Post,Reply
from django.db import transaction

if __name__=='__main__':
	reload(sys)
	sys.setdefaultencoding('utf-8')

	# django will set env['TZ'] according to the settings.py of django project
	logging.info('Time Zone: %s',os.environ['TZ']) #	os.environ['TZ']='Asia/Shanghai'

	pg=CdPostGetter(os.path.join(os.path.dirname(os.path.abspath(__file__)),'cd_cookie.dat'))
	pg.forum_name='cd'
##	pg.test('http://lt.cjdby.net/thread-1145528-1-1.html')
##	sys.exit()

	pg.doWork('cd','cd-生活茶馆',
####								datetime.datetime.today()-datetime.timedelta(minutes=60),
####								datetime.datetime.today()-datetime.timedelta(hours=5),
								datetime.datetime.today()-datetime.timedelta(days=1),
####								datetime.datetime.today()-datetime.timedelta(hours=72),
								1,100)

	pg.doWork('cd','cd-中外历史(原创讨论)',
##								datetime.datetime.today()-datetime.timedelta(minutes=60),
##								datetime.datetime.today()-datetime.timedelta(hours=5),
								datetime.datetime.today()-datetime.timedelta(days=1),
##								datetime.datetime.today()-datetime.timedelta(hours=72),
								1,100)
	logging.debug('done')
	raw_input('press enter to exit ...')
